########################################################################
#  Pyfm - personal finance data-base
#  Copyright (C) 2007,8 Ehud Ben-Reuven
#  udi@benreuven.com
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation version 2.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
########################################################################
"""Fixing functions
checking and fixing individual objects according to the model.
"""
import copy
from obj.flat import stagvalue,line_copy,stv
from obj.obj import Obj,db
from obj.dt import DT
from model import *
from tagsyntax import *
from modellib import *
from trnlib import *
from lib.quoteinterface import getFXrate,getFX
from obj.retype import RE

def fixasset(line):
    """search for problems in Asset objects and try to fix them:
Currency:
- Multiple CURRENCY
- Mulitple ORIGCURRENCY
- Missing ORIGCURRENCY
- Missing CURRENCY 
SECID: warn if an  asset has two different labels from the same domain.
For example: `SECID=CUSIP=012345678` and `SECID=CUSIP=876543210`.
    """
    assets=getassets(line)
    for a in assets:
        assert isinstance(a,Asset)
        # Fix SECID
        #=========
        # Warn about assets that have multiple SECID labels in the same domain
        domains={}
        for s in a.SECID:
            domain,subdomain,label=s.explode()
            if domain=='OTHER': continue # OTHER domain can go crazy
            if subdomain: domain += '='+subdomain
            domains.setdefault(domain,[]).append(label)
        for domain in domains:
            if len(domains[domain])>1:
                pwarn('Multiple labels for domain '+stagvalue(domain,domains[domain]),a)
                
        # Fix ASSETCLASS
        #================
        if db.getalllines({'Asset':[a],'?RATING':[]}) or db.getalllines({'Asset':[a],'DTCOUPON':[]}):
            a.update_tag('ASSETCLASS',[ASSETCLASS('BOND')])

        # Fix CURRENCY
        #============
        # check if the SECID dictates the currency and original currency.
        xcur=None
        xorigcur=None
        for s in a.SECID:
            if s.startswith('X='):
                xorigcur=s[2:5]
                xcur=s[5:]
                if xcur not in a.CURRENCY: a.CURRENCY.append(xcur)
                if xorigcur not in a.ORIGCURRENCY: a.ORIGCURRENCY.append(xorigcur)
                break

        if len(a.CURRENCY)>1:
            perr('Multiple '+stagvalue('CURRENCY',a.CURRENCY),a)
            if xcur:
                pinfo('->'+stagvalue('CURRENCY',xcur)+' because it was in SECID')
                a.CURRENCY=[xcur]
            else:
                cur=db.getall('CURRENCY',{'Asset':[a],'!?DTASOF':[''],'!?DTSETTLE':[''],'!?DTTRADE':['']})
                if len(cur)==1:
                    pinfo('->'+stagvalue('CURRENCY',cur)+' because it was in a dateless line')
                    a.CURRENCY=cur
                else:
                    pinfo('->Can''t fix it!!!')
        elif not a.CURRENCY:
            pinfo('No CURRENCY',a)
            if a.ORIGCURRENCY:
                pinfo('->'+stagvalue('ORIGCURRENCY',a.ORIGCURRENCY))
                a.CURRENCY=a.ORIGCURRENCY
            else:
                pinfo('->No ORIGCURRENCY '+stagvalue('CURRENCY','USD')+stagvalue('ORIGCURRENCY','USD'))
                a.ORIGCURRENCY=[ORIGCURRENCY('USD')]
                a.CURRENCY=a.ORIGCURRENCY
        if len(a.ORIGCURRENCY)>1:
            perr('Multiple '+stagvalue('ORIGCURRENCY',a.ORIGCURRENCY),a)
            if xorigcur:
                pinfo('->'+stagvalue('ORIGCURRENCY',xorigcur)+' because it was in SECID')
                a.ORIGCURRENCY=[xorigcur]
            else:
                pinfo('->Can''t fix it!!!')
        elif not a.ORIGCURRENCY:
            pinfo('No ORIGCURRENCY',a)
            if a.CURRENCY:
                pinfo('->'+stagvalue('CURRENCY',a.CURRENCY))
                a.ORIGCURRENCY=a.CURRENCY
            else:
                pinfo('->no CURRENCY Can''t fix it!!!')

def fixhistory(line):
    """search for problems in History objects (price of assets) and try to fix them:
f Class=History [Asset={}] [History={}]
    """
    maxperr=0.1
    for a in getassets(line):
        historyfilter=getfilter(line,'History')
        add2line(historyfilter,'!DTASOF','SORT')
        historyfilter['Asset']=[a]
        lstphst=None
        if a.FIXEDPRICE:
            lstp=a.FIXEDPRICE[0]
        else:
            lstp=None
        for hst in  db.getallobjects(History,historyfilter):
            if hst.UNITPRICE:
                p=hst.UNITPRICE[0]
                dt=hst.DTASOF[0]
                if p<0:
                    perr('Negative price',hst)
                    continue
                if p>0 and lstp==0.:
                    perr('Price for asset that is not traded any more',hst)
                if a.CURRENCY and a.ORIGCURRENCY and a.CURRENCY[0]!=a.ORIGCURRENCY[0]:
                    p*=getFXrate(dt,a.CURRENCY[0],a.ORIGCURRENCY[0])
                if a.ischecking() and abs(p-1.)>maxperr:
                    perr('Checking Asset with price different than 1',hst)
                if a.isfixedprice() and lstp and abs(p-lstp)>maxperr:
                    perr('Fixed price asset with variable price',hst)
                lstphst=hst
                lstp=p
        if a.isfixedprice() and lstp and not a.FIXEDPRICE:
            a.FIXEDPRICE=[lstp]

def fixholding(line):
    """search for problems in Holding objects and try to fix them"""
    for h in getholdings(line):
        currency=reduce(lambda x,y:x+y,[a.CURRENCY for a in h.Asset],[])
        currency=list(set(currency))
        if len(currency)>1: perr('ambiguous currency'+str(currency),h)
        origcurrency=reduce(lambda x,y:x+y,[a.ORIGCURRENCY for a in h.Asset],[])
        origcurrency=list(set(origcurrency))
        if len(origcurrency)>1: perr('ambiguous original currency'+str(origcurrency),h)
        fixedp=h.getfixedprice()
        if fixedp:
                for a in h.Asset:
                        if a.FIXEDPRICE and abs(fixedp-a.FIXEDPRICE[0])>0.01:
                            perr('%s has %s != %s'%(sobj(a),stv(a,'FIXEDPRICE'),stagvalue('FIXEDPRICE',fixedp)),h)
   
def fixaccountsource(line):
    """search for problems in AccountSource objects and try to fix them"""
    days=int(line.pop('DAYS',['20'])[0])
    grace=1
    min_range=2
    accountsource_filter=getfilter(line,'AccountSource')
    aslist = db.getallobjects(AccountSource,accountsource_filter)

    for accsrc in aslist:
        filter={}
        db.build_filter('HoldingSource',{'Account':accsrc.Account,'FILE':accsrc.FILE},filter)

        # Check dates
        dtstart=accsrc.DTSTART[0] if accsrc.DTSTART else None
        dtend=accsrc.DTEND[0] if accsrc.DTEND else None
        if dtstart and dtend:
            if dtstart>dtend:
                perr('%s is after %s'%(stagvalue('DTSTART',dtstart),stagvalue('DTEND',dtend)),accsrc)
            elif dtstart>=dtend-min_range:
                pwarn('%s is less than %d days before %s'%(stagvalue('DTSTART',dtstart),min_range,stagvalue('DTEND',dtend)),accsrc)
        if dtstart or dtend:
            dates_found=False
            for tag in ['DTASOF','DTSETTLE','DTTRADE']:
                dts=db.getall(tag,filter)
                if dts:
                    dates_found=True
                else:
                    continue
                    
                if dtend:
                    maxdt=max(dts)
                    if maxdt>dtend:
                        if maxdt<=dtend+grace:
                            pinfo('Max %s after %s'%(stagvalue(tag,maxdt),stagvalue('DTEND',dtend)),accsrc)
                        else:
                            perr('Max %s after %s'%(stagvalue(tag,maxdt),stagvalue('DTEND',dtend)),accsrc)
                    elif maxdt<dtend-days:
                        pwarn('Max %s is %d days before %s'%(stagvalue(tag,maxdt),days,stagvalue('DTEND',dtend)),accsrc)    
                if dtstart:
                    mindt=min(dts)
                    if mindt<dtstart:
                        if mindt>=dtstart-grace:
                            pinfo('Min %s before %s'%(stagvalue(tag,mindt),stagvalue('DTSTART',dtstart)),accsrc)
                        elif tag=='DTTRADE':
                            pwarn('Min %s before %s'%(stagvalue(tag,mindt),stagvalue('DTSTART',dtstart)),accsrc)
                        else:
                            perr('Min %s before %s'%(stagvalue(tag,mindt),stagvalue('DTSTART',dtstart)),accsrc)
                    elif mindt>dtstart+days and tag!='DTASOF':
                        pwarn('Min %s is %d days after %s'%(stagvalue(tag,mindt),days,stagvalue('DTSTART',dtstart)),accsrc)                    
            if not dates_found:
                pinfo('No dates where found',accsrc)
                
def fixbuysell(t,posunits,v,value_error,fix):
    merged=trnmerged(t)
    
    if t.UNITPRICE:
        p=t.UNITPRICE[0]
    else:
        p=0.
    if not (t.UNITS and t.UNITS[0]):
        if v: perr('Missing units',t)
        u=None
        if merged: posunits=None
    else:
        u=t.UNITS[0]
        if merged and posunits!=None: posunits+=u
    if merged and posunits!=None and posunits<0:
        if v: pwarn('Negative position %f'%posunits,t)

    # ACCRDINT
    if t.ACCRDINT and t.ACCRDINT[0]:
        if t.INTEREST and t.INTEREST[0]:
            if v: perr('INTEREST and ACCRDINT in BUY/SELL %f %f'%(t.ACCRDINT[0],t.INTEREST[0]),t)
        else:
            if v: pinfo('Switching ACCRDINT to INTEREST',t)
            if fix: t.INTEREST=t.ACCRDINT
        if fix: t.ACCRDINT=[]                       
    # INTEREST
    if t.INTEREST and t.INTEREST[0]:
        trntype=t.TRNTYPE[0]
        if trntype=='BUY' and t.INTEREST[0]>0:
            if v: perr("Buy contained positive interest",t)
        if trntype=='SELL' and t.INTEREST[0]<0:
            if v: perr("Sell contained negative interest",t)
        if v: pinfo("Purchase contained interest (assuming it's accrued interest)",t)
    # COST
    if u and p:                   
        if t.COST:
            cost=t.COST[0]
            if cost*u<0:
                if v: pwarn('Cost has negative sign',t)
                cost=-cost
            cost_err=cost-u*p
            if abs(cost_err/(u*p))>0.15:
                raise Exception('COST way different from real-cost')
            if abs(cost-u*p)>value_error:
                # Check if difference can be an ACCRDINT
                if not t.INTEREST and cost_err*u>0:
                    ac=t.HoldingSource[0].Holding[0].Asset[0].ASSETCLASS
                    if any(a=='BOND' for a in ac ) and cost_err*u<0:
                        if v: pwarn('Cost diff %f taken to be ACCRDINT'%cost_err,t) 
                        if fix: t.INTEREST=[cost_err]
                    else:
                        if v: perr('Cost %f does not match by %f'%(cost,cost_err),t)
                else:
                    if v: perr('Cost %f does not match by %f'%(cost,cost_err),t)
        if fix: t.COST=[u*p]
    computed_cost=trnincome(t) #  TOTAL-expense
    if computed_cost!=None:
        if t.INTEREST and t.INTEREST[0]: computed_cost-=t.INTEREST[0]
        computed_cost=-computed_cost
    if t.COST and computed_cost!=None and abs(t.COST[0]-computed_cost)>value_error:
        if v: perr('Cost does not match',t)
    if computed_cost!=None: t.COST=[computed_cost]
    # compute UNITPRICE
    if not t.UNITPRICE:
        if u and t.COST:
            if v: pwarn('Missing price',t)
            if fix: t.UNITPRICE=[t.COST[0]/u]
        else:
            if v: perr('Unknwon price',t)
    return posunits

def fixtransaction(line={}):
    """f Class=Transaction [Holding={}] [Transaction={}] [DTSTART=PAST] [FIX=TRUE] [VALUE_ERROR=1] [DAYS_ERROR=30]
Test validity of transactions.
Report transactions that are after DTSTART
If FIX, try to fix some of the problems

ft Transaction=|FILE!=_MERGED_|
check all non-merged transactions
"""
    # TODO: checking transactions in which the PAYEEID is ACCRDINT should be related (RELFITID) to BUY or SELL of debit transactions and UNITS>0 on SELL UNITS<0 on BUY
    dtstart=line.pop('DTSTART',[DTASOF('PAST')])[0]
    fix=line.pop('FIX',[True])[0]
    value_error=float(line.pop('VALUE_ERROR',['1.'])[0])
    dtsettle_error=int(line.pop('DAYS_ERROR',['30'])[0])
    transaction_filter=gettransaction_filter(line,default={})
    for h in getholdings(line):
        assert isinstance(h,Holding)
        trans=gettransactions(transaction_filter,h)
        if not trans: continue
        # Merge transactions from same holding-source and same FITID/RELFITID
        if fix:
            for t in trans:
                if trnmerged(t): continue
                # If a related transaction is in the same holding as this transaction (t) then t's content should be merged into the related transaction
                # This can happen when an import tool generate several lines for each imported transactions without beeing aware that the two splits
                # are in the same holding-source
                for fitid in t.RELFITID:
                    relt=db.getallobjects('Transaction',{'FITID':[fitid],'HoldingSource':t.HoldingSource})
                    if not relt: continue
                    if len(relt)>1: raise Exception("More than one transaction in same source with same FITID")
                    if trnskip(t): raise Exception("Transaction already paired with another transacion")
                    relt=relt[0]
                    for tag in t.tags:
                        if tag in ['RELFITID','_OTran_','_MTran_','HoldingSource']: continue
                        totag=tag
                        if tag in ['NLINE']:
                            totag='OLINE'
                        relt.update_tag(totag,t[tag])
                    t._SKIP_=[True]
        # DTSETTLE
        for t in trans:
            if not t.DTSETTLE:
                raise Exception('No DTSETTLE')
            if len(t.DTSETTLE)>1:
                perr('Multiple %s'%stagvalue('DTSETTLE',t.DTSETTLE),t)
                if fix: t.DTSETTLE=t.DTSETTLE[:1]
        trans.sort(key=lambda t: t.dttrade()) #oldest first.
        #
        currency=None # Currency used in all transactions of this holding
        posunits=0 # Position of the holding after every transaction
        # For merged transactions, Check if two merged transactions have the same FITID
        previous_merged_fitid={} # {OFITID:t}
        for idxt,t in enumerate(trans):
            if trnskip(t): continue
            # Check dates
            dt=t.DTSETTLE[0]
            v = dt>=dtstart
            # DTTRADE
            if t.DTTRADE:
                if len(t.DTTRADE)>1:
                    if v: perr('Multiple %s'%stagvalue('DTTRADE',t.DTTRADE),t)
                    if fix: t.DTTRADE=t.DTTRADE[:1]
                dttrade=t.DTTRADE[0]
                if dttrade>dt:
                    tdays=dttrade-dt
                    if tdays>dtsettle_error:
                        if v: pwarn('DTTRADE %d after DTSETTLE'%tdays,t)
            elif trnmerged(t):
                if fix:
                    t.update_tag('DTTRADE',t.DTSETTLE)
            # UNITPRICE & CURRENCY
            if len(t.UNITPRICE)>1:
                if v: perr('Multiple %s'%stagvalue('UNITPRICE',t.UNITPRICE),t)
                if fix: t.UNITPRICE=t.UNITPRICE[:1]
            if t.UNITPRICE and t.UNITPRICE[0]<0:
                if v: perr('UNITPRICE<0',t)
                if fix: t.UNITPRICE=[]
            if t.UNITPRICE and t.UNITPRICE[0]==0: # IN/OUT
                #msg.append('UNITPRICE=0')
                if fix: t.UNITPRICE=[]
            # MARKUP/DOWN
            if t.MARKUP and not t.UNITPRICE:
                if v: perr('Markup without units',t)
            if t.MARKDOWN and not t.UNITPRICE:
                if v: perr('Markup without units',t)
            if t.MARKUP and t.MARKUP[0]<0:
                if v: perr('Negative markup',t)
            if t.MARKDOWN and t.MARKDOWN[0]<0:
                if v: perr('Negative markdown',t)
            # Effective price&currency
            p,cur=getunitpricecur(t)
            if not p: p=0.
            if p<0:
                if v: perr('Negative unit price %f'%p,t)
                p=0.
            if p and not t.UNITPRICE: # p might be different because of MARKUP/DOWN
                if fix: t.UNITPRICE=[UNITPRICE(p)]
            if cur:
                if fix: t.CURRENCY=[cur]
                if not currency: currency=cur
                if currency!=cur:
                    if v: perr('Curency change',t)
            if h.isfixedprice():
                origcur=None
                for a in h.Asset:
                    if a.ORIGCURRENCY:
                        origcur=a.ORIGCURRENCY[0]
                        break
                xrate=None
                if cur and origcur:
                    xrate,dtxrate=getFX(dt,origcur,cur) # transaction UNITPRICE is the convertion from ORIGINALCURRENCY to the transaction's CURRENCY
                if h.ischecking():
                    if not p: p=xrate
                    if abs(p-xrate)>0.01:
                        if p==1. or abs(p/xrate-1)>0.2:
                            perr('%s does not match FX %f'%(stagvalue('UNITPRICE',p),xrate),t)
                            if fix: p=xrate
                        else:
                            pwarn('%s does not match FX %f'%(stagvalue('UNITPRICE',p),xrate),t)
                    if fix and p: t.UNITPRICE[0]=UNITPRICE(p)
                else: # isfixedprice
                    if t.UNITPRICE:
                        fixedp=t.UNITPRICE[0] # dont use p as it may have a markup/down
                        fixedp/=xrate # convert from transaction currency to original currency
                        for a in h.Asset:
                            if not a.FIXEDPRICE:
                                a.FIXEDPRICE=[fixedp]
                            else:
                                if abs(fixedp-a.FIXEDPRICE[0])>0.01:
                                    perr('%s does not match %s'%(stagvalue('UNITPRICE',p),stv('FIXEDPRICE',a)),t)
                    else:
                        fixedp,dtdummy=getCheckingPrice(h,cur,dt)
                        if fix and fixedp: t.UNITPRICE=[UNITPRICE(fixedp)]
            # TRNTYPE
            if not t.TRNTYPE:
                trntype=None
                if h.ischecking():
                    if t.UNITS and t.UNITS[0]>0:
                        trntype='CREDIT'
                    elif t.UNITS and t.UNITS[0]<0:
                        trntype='DEBIT'
                else:
                    if t.UNITS and t.UNITS[0]>0:
                        trntype='BUY'
                    elif t.UNITS and t.UNITS[0]<0:
                        trntype='SELL'
                    elif t.INTEREST:
                        trntype='INT'
                        if v: pwarn('Unknwon TRNTYPE using '+trntype,t)
                if not trntype:
                    trntype='MISC'                    
                    if v: pwarn('Unknwon TRNTYPE using '+trntype,t)
                if fix: t.TRNTYPE=[trntype]
            else:
                trntype = t.TRNTYPE[0]
            if trntype=='OPENING_BALANCE':
                if h.ischecking():
                    for tt in trans:
                        if (tt!=t and 
                            tt.HoldingSource[0].FILE[0] == t.HoldingSource[0].FILE[0] and
                            tt.DTSETTLE[0] <= t.DTSETTLE[0]):
                            perr('Opening balance is not first',t)
                else:
                    perr('Opnening balance in non checking',t)
                
                
            # ACCRDINT
            if t.ACCRDINT and not t.ACCRDINT[0]:
                if fix: t.ACCRDINT=[]
            if t.ACCRDINT and not (t.UNITS and t.UNITS[0]):
                pwarn("ACCRDINT not on buy/sell",t)
                if fix: t.ACCRDINT=[]
                
            
                
            # Handle Checking transactions
            #==============================
            if h.isfixedprice():
                if trntype in ['BUY','SELL']:
                    if v: pwarn('Inappriopriate %s'%stagvalue('TRNTYPE',trntype),t)
                # All other transaction numerical-components are not relevant for a Checking transaction.
                for tag in ['LOAD','MARKUP','MARKDOWN','ACCRDINT']:
                    if t[tag] and t[tag][0]:
                        if v: perr('Transaction in checking account with '+stv(t,tag),t)
                if not t.UNITS:
                    if t.TOTAL and p:
                        if v: pinfo('no UNITS',t)
                        if h.ischecking():
                            # In Checking holdingg TOTAL is how much money got into that holding
                            if fix: t.UNITS=[UNITS(t.TOTAL[0]/p)]
                        else:
                            # all other holdings (including deposits that are fixed price) TOAL is how much money transfered into another
                            # checking holding
                            if fix: t.UNITS=[UNITS(-t.TOTAL[0]/p)]
                    else:
                        if v: perr('no UNITS',t)

            # Handle Equity transactions
            #==============================
            elif trntype in ['TRANSFER','IN','OUT']: #FIXME handle transfer between holdings/accounts
                # If the following transactions is balancing out this transaction thenignore both.
                if idxt<len(trans)-1:
                    nextt=trans[idxt+1]
                    if t.UNITS and nextt.UNITS and t.UNITS[0]==-nextt.UNITS[0]:
                        t.UNITS[0]=0.
                        nextt.UNITS[0]=0.
                        if nextt.TRNTYPE and nextt.TRNTYPE[0] not in ['TRANSFER','IN','OUT']:
                            perr('Following transaction is not a transfer',nextt)
                    elif idxt!=0:
                        perr('Dont know how to handle such transfers',t)
                    
                for tag in ['INTEREST','COST','LOAD','MARKUP','MARKDOWN','ACCRDINT']:
                    if t[tag] and t[tag][0]:
                        if v: perr('Transfer with '+stv(t,tag),t)
                if not t.UNITS:
                    if v: perr('Transfer does not contain units',t)
            elif trntype in ['BUY','SELL']:
                posunits=fixbuysell(t,posunits,v,value_error,fix)
            elif trntype in ['INT','DIV','CG','CGLONG','CGSHORT']:
                if t.COST and t.COST[0]:
                    if t.INTEREST and t.INTEREST[0]:
                        if v: perr('Interest and Cost togther',t)                        
                    else:
                        if v: pinfo('Replacing cost with interest',t)
                        income=-t.COST[0]
                        if fix: t.INTEREST=[-income]
                        if fix: t.COST=[]
                if not t.INTEREST:
                    income=trnincome(t)    
                    if income!=None:
                        if v: pinfo('%s without interest'%trntype,t)
                        if fix: t.INTEREST=[income]
                    elif t.ACCRDINT:
                        if fix: t.INTEREST=[0.]
                    else:
                        if v: perr('%s without interest'%trntype,t)
                if t.UNITS and t.UNITS[0]:
                    if v: pwarn('Reinvestment in the same transaction',t)
                    posunits=fixbuysell(t,posunits,v,value_error,fix)
                else:
                    for tag in ['LOAD']:
                        if t[tag] and t[tag][0]:
                            if v: perr('Inappriopriate %s with %s'%(stagvalue('TRNTYPE',trntype),stv(t,tag)),t)
            elif trntype in ['EXPENSE','TAX','FEE']: # Expense is usually a seperate line for tax or fees.
                if (t.COST and t.COST[0]) or (t.INTEREST and t.INTEREST[0]) or (t.UNITS and t.UNITS[0]):
                    if v: perr('COST/INTEREST/UNITS in with %s'%(stagvalue('TRNTYPE',trntype)),t)
                # Check if there is unexplained expense (explained by TAXES, FEES, ...) in the TOTAL of the transaction
                # and if there is, try to plug it in COMMISSION
                expense=trnincome(t)
                if expense: # check that it is not None or zero
                    expense=-expense
                    if not t.COMMISSION:
                        if fix: t.COMMISSION=[expense]
                    else:
                        if v: perr('Dont have where to place expense %f'%expense,t)
            else:
                if  t.TOTAL or trntotal(t)!=None:
                    if v: perr('Inappriopriate %s'%stagvalue('TRNTYPE',trntype),t)
            # TOTAL
            computed_total=trntotal(t)
            if t.TOTAL and computed_total!=None and abs(computed_total-t.TOTAL[0])>value_error:
                if t.FEES and abs(computed_total-(t.TOTAL[0]-t.FEES[0]))<=value_error:
                    if v: pinfo('%s misses %s'%(stagvalue('TOTAL',t.TOTAL),stagvalue('FEES',t.FEES)),t)
                else:
                    if v: pwarn('%s differ from computed by %f'%(stagvalue('TOTAL',t.TOTAL),t.TOTAL[0]-computed_total),t)
            if computed_total!=None:
                if fix: t.TOTAL=[computed_total]
                
            # Fix special attributes of merged transactions
            #===============================================
            if trnmerged(t):
                # Check if two different merged transactions have an overlap in their FITID
                for fitid in t.OFITID:
                    if fitid in previous_merged_fitid and previous_merged_fitid[fitid]!=t:
                        perr('%s with same %s'%(stagvalue('Transaction',previous_merged_fitid[fitid]),fitid),t)
                        raise Exception("Overlap in FITID of merged transactions")
                    else:
                        previous_merged_fitid[fitid]=t
def reverse_total(line):
    if 'TOTAL' in line: line['TOTAL'][0]*=-1

def fixXtransaction(param_line={}):
    """f Class=Xtran [Holding={}] [Transaction={}] [DTSTART=PAST]
"""
    dtstart=param_line.pop('DTSTART',[DTASOF('PAST')])[0]
    transaction_filter=gettransaction_filter(param_line,default={})
    for h in getholdings(param_line):
        assert isinstance(h,Holding)
        trans=gettransactions(transaction_filter,h)
        if not trans: continue
        for t in trans:
            line={}
            for attr in ['ACCTID','SECNAME','TRNTYPE','SECID','UNITS']:
                xattr='X'+attr
                if t[xattr]: line[attr]=t[xattr]
            if not line: continue
            xline=None
            line['FILE']=t.HoldingSource[0].FILE
            for attr in ['DTSETTLE','DTTRADE','MEMO','FITID','CURRENCY','TOTAL']:
                if t[attr]: line[attr]=t[attr] 
            #if t.TOTAL: line['TOTAL']=[-t.TOTAL[0]]
            if 'CURRENCY' not in line:
                h=t.HoldingSource[0].Holding[0]
                if h.Account:
                    line['CURRENCY']=h.Account[0].CURDEF
            if 'ACCTID' in line:
                reverse_total(line)
                acct=db.getallobjects('Account',{'ACCTID':line['ACCTID']})
                for accttype in acct[0].ACCTTYPE:
                    if accttype.iscurrent():
                        line['SECID']=[SECID('CURRENT='+accttype+'='+acct[0].CURDEF[0])]
                        if 'TRNTYPE' not in line:
                            pwarn('Missing TRNTYPE',line)
                            if line['TOTAL'][0]<0:
                                line['TRNTYPE']=['DEBIT']
                            else:
                                line['TRNTYPE']=['CREDIT']
                        if line['TRNTYPE'][0]=='INT':
                            xline=line_copy(line)
                            xline['TRNTYPE']=['TRANSFER']
                            reverse_total(line)
                        elif line['TRNTYPE'][0] not in ['CREDIT','DEBIT']:
                            perr('In appropriate TRNTYPE %s for current account'%line['TRNTYPE'][0])
                        break
            else:
                # If we dont know in which account the cross transaction is
                # then generate a generic account. Without an account there
                # is no holding or transaction
                h=t.HoldingSource[0].Holding[0]
                if h.Account:
                    line['ACCTGRP']=h.Account[0].ACCTGRP
                    line['CURDEF']=h.Account[0].CURDEF
                    line['ACCTTYPE']=[ACCTTYPE('INVESTMENT')]
                # A transaction must have a SECID so make sure we have one.
                fixlineSECID(line)
            trnXadd(t,line)
            if xline: trnXadd(t,xline)

def fixlineSECID(line):
    if 'SECID' in line: return False 
    if 'SECNAME' not in line: return False
    asset=db.getallobjects('Asset',{'SECNAME':line['SECNAME']})
    if asset:
        if len(asset)>1:
            perr('Too many assets matches SECNAME=%s'%line['SECNAME'][0],asset)
            return False
        else:
            asset[0].build_line(line)
            return True
    asset=db.getallobjects('Asset',{'SECNAME':[RE(s) for s in line['SECNAME']]})
    if not asset:
        perr('No asset matches SECNAME=/%s/'%line['SECNAME'][0])
    elif len(asset)>1:
        perr('Too many assets matches SECNAME=/%s/'%line['SECNAME'][0],asset)
    else:
        asset[0].build_line(line)
        return True
    return False
     
def fixSECID(filter={}):
    """f Class=SECID [<line-filter>]
Find lines with a SECNAME but without a specific SECID and try to deduce it.
"""
    for line in db.getall('Line',filter):
        if fixlineSECID(line):
            # We added new tags to the line, so now added all objects that may 
            # result from these new tags
            db.line_clsupdate(line)

def do_f(self,line={}):
    """Model fix command:
f Class=* <parameters>"""
    classes=line.pop('Class',['AccountSource','Asset','History','Holding','Transaction'])
    for cls in classes:
        pinfo('Checking/Fixing '+cls)
        l=line_copy(line)
        if cls=='AccountSource':
            fixaccountsource(l)
        elif cls=='Asset':
            fixasset(l)
        elif cls=='History':
            fixhistory(l)
        elif cls=='Holding':
            fixholding(l)
        elif cls=='Transaction':
            fixtransaction(l)
        elif cls=='Xtran':
            fixXtransaction(l)
        elif cls=='SECID':
            fixSECID(l)
        else:
            print "Unknown class",cls
            
